{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 采用 xgboost 模型完成商品分类"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 第二步：调整树的参数：max_depth & min_child_weight\n",
    "(粗调，参数的步长为2；下一步是在粗调最佳参数周围，将步长降为1，进行精细调整)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "首先导入必要的工具包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###   数据读取\n",
    "注：数据已经提前被处理，只需根据作业要求读取 RentListingInquries_FE_train.csv 文件即可"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# path to where the data lies\n",
    "path = './data/' \n",
    "fileName = 'RentListingInquries_FE_train.csv'\n",
    "\n",
    "data=pd.read_csv(path+fileName)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Target 分布，看看各类样本分布是否均衡"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAEKCAYAAADaa8itAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAGlRJREFUeJzt3X+0XWV95/H3hwBK/ZUgV1ZKoNBO2pHaNmqKUWZaRQuBTgUd7cByJGNZk9oBi2s6XWKnU1TE1rZqa0fp0GUKdKyR+otoY9OUio5WfgRFICBDilYiGYiGnzriAN/54zy3nISbe3fiPjk5yfu11l5n7+9+9j7fs+5Kvmvv59nPTlUhSVIfDhh3ApKkfYdFRZLUG4uKJKk3FhVJUm8sKpKk3lhUJEm9sahIknpjUZEk9caiIknqzYHjTmBPO+yww+roo48edxqSNFGuv/76b1XV1Fzt9ruicvTRR7Nhw4ZxpyFJEyXJP3Vp5+0vSVJvRlZUkjw5ybVJvpJkY5K3tvglSb6W5Ia2LGnxJHlvkk1JbkzyvKFzrUhye1tWDMWfn+Smdsx7k2RUv0eSNLdR3v56GDihqh5KchDw+SSfbvt+s6o+skP7k4HFbXkBcBHwgiSHAucDS4ECrk+ypqrubW1WAlcDa4HlwKeRJI3FyK5UauChtnlQW2abZ/9U4LJ23NXA/CQLgZOA9VW1rRWS9cDytu/pVfXFGszffxlw2qh+jyRpbiPtU0kyL8kNwD0MCsM1bdeF7RbXe5I8qcWOAO4cOnxzi80W3zxDfKY8VibZkGTD1q1bf+DfJUma2UiLSlU9WlVLgEXAcUmeA7wZ+JfAzwKHAm9qzWfqD6ndiM+Ux8VVtbSqlk5NzTkiTpK0m/bI6K+qug+4ClheVVvaLa6HgT8HjmvNNgNHDh22CLhrjviiGeKSpDEZ5eivqSTz2/ohwMuAr7a+ENpIrdOAm9sha4Az2yiwZcD9VbUFWAecmGRBkgXAicC6tu/BJMvauc4ErhjV75EkzW2Uo78WApcmmcegeF1eVZ9K8vdJphjcvroBeH1rvxY4BdgEfBd4HUBVbUtyAXBda/e2qtrW1n8NuAQ4hMGoL0d+SdIYZTBwav+xdOnS8ol6ae92/J8cP+4U9nlfeMMXdql9kuuraulc7XyiXpLUG4uKJKk3FhVJUm8sKpKk3lhUJEm9sahIknpjUZEk9caiIknqjUVFktQbi4okqTcWFUlSbywqkqTeWFQkSb2xqEiSemNRkST1xqIiSeqNRUWS1BuLiiSpNxYVSVJvLCqSpN5YVCRJvRlZUUny5CTXJvlKko1J3trixyS5JsntST6c5OAWf1Lb3tT2Hz10rje3+G1JThqKL2+xTUnOG9VvkSR1M8orlYeBE6rqZ4AlwPIky4B3Au+pqsXAvcBZrf1ZwL1V9S+A97R2JDkWOB34SWA58P4k85LMA94HnAwcC5zR2kqSxmRkRaUGHmqbB7WlgBOAj7T4pcBpbf3Utk3b/9IkafHVVfVwVX0N2AQc15ZNVXVHVX0fWN3aSpLGZKR9Ku2K4gbgHmA98I/AfVX1SGuyGTiirR8B3AnQ9t8PPHM4vsMxO4tLksZkpEWlqh6tqiXAIgZXFs+eqVn7zE727Wr8CZKsTLIhyYatW7fOnbgkabfskdFfVXUfcBWwDJif5MC2axFwV1vfDBwJ0PY/A9g2HN/hmJ3FZ/r+i6tqaVUtnZqa6uMnSZJmMMrRX1NJ5rf1Q4CXAbcCnwFe1ZqtAK5o62vaNm3/31dVtfjpbXTYMcBi4FrgOmBxG012MIPO/DWj+j2SpLkdOHeT3bYQuLSN0joAuLyqPpXkFmB1krcDXwY+0Np/APiLJJsYXKGcDlBVG5NcDtwCPAKcXVWPAiQ5B1gHzANWVdXGEf4eSdIcRlZUqupG4LkzxO9g0L+yY/x7wKt3cq4LgQtniK8F1v7AyUqSeuET9ZKk3lhUJEm9sahIknpjUZEk9caiIknqjUVFktQbi4okqTcWFUlSbywqkqTeWFQkSb2Zs6gkeUqSA9r6jyd5eZKDRp+aJGnSdLlS+Rzw5CRHAFcCrwMuGWVSkqTJ1KWopKq+C7wS+JOqegWDd8JLkrSdTkUlyQuB1wB/3WKjnDJfkjShuhSVNwJvBj7e3m3yowxetCVJ0nbmvOKoqs8Cn03ylLZ9B/Dro05MkjR5uoz+emF7W+Otbftnkrx/5JlJkiZOl9tffwScBHwboKq+AvzcKJOSJE2mTg8/VtWdO4QeHUEukqQJ12UU151JXgRUkoMZ9KfcOtq0JEmTqMuVyuuBs4EjgM3AkrYtSdJ25iwqVfWtqnpNVR1eVc+qqn9fVd+e67gkRyb5TJJbk2xMcm6LvyXJN5Pc0JZTho55c5JNSW5LctJQfHmLbUpy3lD8mCTXJLk9yYfblZQkaUy6jP66NMn8oe0FSVZ1OPcjwG9U1bOBZcDZSaafxH9PVS1py9p23mOB04GfBJYD708yL8k84H3AyQye5D9j6DzvbOdaDNwLnNUhL0nSiHS5/fXTVXXf9EZV3Qs8d66DqmpLVX2prT/IoB/miFkOORVYXVUPV9XXgE3AcW3ZVFV3VNX3gdXAqUkCnAB8pB1/KXBah98jSRqRLkXlgCQLpjeSHMouTtOS5GgGheiaFjonyY1JVg2d+whgeJTZ5hbbWfyZwH1V9cgOcUnSmHQpKu8C/iHJBUkuAP4B+P2uX5DkqcBHgTdW1QPARcCPMejw39LOD5AZDq/diM+Uw8okG5Js2Lp1a9fUJUm7qEtH/WXAq4C7gXuAV1bVX3Q5eXvvykeBD1bVx9r57q6qR6vqMeDPGNzegsGVxpFDhy8C7pol/i1gfpIDd4jP9BsurqqlVbV0amqqS+qSpN3Q9c2PXwU+BlwBPJTkqLkOaH0eHwBurap3D8UXDjV7BXBzW18DnJ7kSUmOARYD1wLXAYvbSK+DGXTmr6mqYjCx5ava8StafpKkMZmzbyTJG4DzGVypPMrgtlMBPz3HoccDrwVuSnJDi/0Wg9FbS9o5vg78KkCbAfly4BYGI8fOrqpHWw7nAOuAecCqqtrYzvcmYHWStwNfZlDEJElj0qXD/VzgJ7o8mzKsqj7PzP0ea2c55kLgwhnia2c6rs2YfNyOcUnSeHS5/XUncP+oE5EkTb4uVyp3AFcl+Wvg4engcD+JJEnQrah8oy0Ht0WSpBl1efPjWwGSPKWqvjP6lCRJk8o3P0qSeuObHyVJvfHNj5Kk3vjmR0lSb3zzoySpN7NeqbQXZL22ql6zh/KRJE2wWa9U2txbp+6hXCRJE65Ln8oXkvx34MPAPz+nMv1WR0mSpnUpKi9qn28bihWDV/lKkvTP5upTOQC4qKou30P5SJIm2Fx9Ko8B5+yhXCRJE67LkOL1Sf5LkiOTHDq9jDwzSdLE6dKn8ivtc/jZlAJ+tP90JEmTrMssxcfsiUQkSZOvyzvqz5wpXlWX9Z+OJGmSdbn99bND608GXgp8CbCoSJK20+X21xuGt5M8A/iLkWUkSZpYnaa+38F3gcVzNWqjxT6T5NYkG5Oc2+KHJlmf5Pb2uaDFk+S9STYluTHJ84bOtaK1vz3JiqH485Pc1I55b5Lsxu+RJPWky5sfP5lkTVs+BdwGXNHh3I8Av1FVzwaWAWcnORY4D7iyqhYDV7ZtgJMZFKvFwErgovb9hwLnAy8AjgPOny5Erc3KoeOWd8hLkjQiXfpU/nBo/RHgn6pq81wHVdUWYEtbfzDJrQymzz8VeHFrdilwFfCmFr+sqgq4Osn8JAtb2/VVtQ0gyXpgeZKrgKdX1Rdb/DLgNODTHX6TJGkEuhSVbwBbqup7AEkOSXJ0VX2965ckORp4LnANcHgrOFTVliTPas2OAIbfMLm5xWaLb54hLkkaky59Kn8FPDa0/WiLdZLkqcBHgTdW1QOzNZ0hVrsRnymHlUk2JNmwdevWuVKWJO2mLkXlwKr6/vRGWz+4y8mTHMSgoHywqj7Wwne321q0z3tafDNw5NDhi4C75ogvmiH+BFV1cVUtraqlU1NTXVKXJO2GLkVla5KXT28kORX41lwHtZFYHwBurap3D+1aA0yP4FrB453+a4Az2yiwZcD97TbZOuDEJAtaB/2JwLq278Eky9p3nUm3AQSSpBHp0qfyeuCD7UVdMLhCmPEp+x0cD7wWuCnJDS32W8DvAZcnOYtBf82r2761wCnAJgbDll8HUFXbklwAXNfavW260x74NeAS4BAGHfR20kvSGHV5+PEfgWWtbyRV9WCXE1fV55m53wMGT+Xv2L7YftLK4X2rgFUzxDcAz+mSjyRp9Lo8p/KOJPOr6qE2NHhBkrfvieQkSZOlS5/KyVV13/RGVd3L4DaVJEnb6VJU5iV50vRGkkOAJ83SXpK0n+rSUf8/gSuT/DmD50B+hcGT8JIkbadLR/3vJ7kReFkLXVBV60abliRpEnW5UgH4MnAQgyuVL48uHUnSJOsy+uuXgWuBVwG/DFyT5FWjTkySNHm6XKn8V+Bnq+oegCRTwN8BHxllYpKkydNl9NcB0wWl+XbH4yRJ+5kuVyp/k2Qd8KG2/e8YTKkiSdJ2uoz++s0krwT+FYNpVy6uqo+PPDNJ0sTpNPqrTVv/sTkbSpL2a/aNSJJ6Y1GRJPVmp0UlyZXt8517Lh1J0iSbrU9lYZKfB16eZDU7vBulqr400swkSRNntqLyO8B5DN79/u4d9hVwwqiSkiRNpp0Wlar6CPCRJP+tqi7YgzlJkiZUl+dULkjycuDnWuiqqvrUaNOSJE2iLhNK/i5wLnBLW85tMUmSttPl4cdfBJZU1WMASS5lMP39m0eZmCRp8nR9TmX+0PozRpGIJGnydSkqvwt8Ockl7SrleuAdcx2UZFWSe5LcPBR7S5JvJrmhLacM7Xtzkk1Jbkty0lB8eYttSnLeUPyYJNckuT3Jh5Mc3PVHS5JGY86iUlUfApYxmPvrY8ALq2p1h3NfAiyfIf6eqlrSlrUASY4FTgd+sh3z/iTzkswD3gecDBwLnNHaAryznWsxcC9wVoecJEkj1On2V1Vtqao1VXVFVf2fjsd8DtjWMY9TgdVV9XBVfQ3YBBzXlk1VdUdVfR9YDZyaJAyek5l+UdilwGkdv0uSNCLjmPvrnCQ3tttjC1rsCODOoTabW2xn8WcC91XVIzvEZ5RkZZINSTZs3bq1r98hSdrBni4qFwE/BiwBtgDvavHM0LZ2Iz6jqrq4qpZW1dKpqaldy1iS1NmsRSXJAcMd7T+oqrq7qh5tw5P/jMHtLRhcaRw51HQRcNcs8W8B85McuENckjRGsxaV9p//V5Ic1ceXJVk4tPkKYLpgrQFOT/KkJMcAi4FrgeuAxW2k18EMOvPXVFUBnwFe1Y5fAVzRR46SpN3X5eHHhcDGJNcC35kOVtXLZzsoyYeAFwOHJdkMnA+8OMkSBreqvg78ajvXxiSXM3hi/xHg7Kp6tJ3nHGAdMA9YVVUb21e8CVid5O0MHsb8QJcfLEkanS5F5a27c+KqOmOG8E7/46+qC4ELZ4ivBdbOEL+Dx2+fSZL2Al0mlPxskh8BFlfV3yX5IQZXDZIkbafLhJL/kcHzIP+jhY4APjHKpCRJk6nLkOKzgeOBBwCq6nbgWaNMSpI0mboUlYfb0+wAtGG8O30mRJK0/+pSVD6b5LeAQ5L8AvBXwCdHm5YkaRJ1KSrnAVuBmxgMAV4L/PYok5IkTaYuo78ea1PeX8Pgttdt7eFDSZK2M2dRSfKLwJ8C/8hgzq1jkvxqVX161MlJkiZLl4cf3wW8pKo2AST5MeCvAYuKJGk7XfpU7pkuKM0dwD0jykeSNMF2eqWS5JVtdWOStcDlDPpUXs1gokdJkrYz2+2vXxpavxv4+ba+FVjwxOaSpP3dTotKVb1uTyYiSZp8XUZ/HQO8ATh6uP1cU99LkvY/XUZ/fYLBlPWfBB4bbTqSpEnWpah8r6reO/JMJEkTr0tR+eMk5wN/Czw8HayqL40sK0nSROpSVH4KeC1wAo/f/qq2Le2VvvG2nxp3CvuFo37npnGnoL1Ml6LyCuBHh6e/lyRpJl2eqP8KMH/UiUiSJl+XK5XDga8muY7t+1QcUixJ2k6XonL+7pw4ySrg3zCYO+w5LXYo8GEGz7x8Hfjlqro3SYA/Bk4Bvgv8h+mBAElW8Pj7W95eVZe2+POBS4BDGLzj5Vyn5Jek8Zrz9ldVfXampcO5LwGW7xA7D7iyqhYDV7ZtgJOBxW1ZCVwE/1yEzgdeABwHnJ9keoqYi1rb6eN2/C5J0h42Z1FJ8mCSB9ryvSSPJnlgruOq6nPAth3CpwKXtvVLgdOG4pfVwNXA/CQLgZOA9VW1raruBdYDy9u+p1fVF9vVyWVD55IkjUmXNz8+bXg7yWkMrhp2x+FVtaWdd0uSZ7X4EcCdQ+02t9hs8c0zxGeUZCWDqxqOOuqo3UxdkjSXLqO/tlNVn6D/Z1Qy01ftRnxGVXVxVS2tqqVTU1O7maIkaS5dJpR85dDmAcBSZvkPfA53J1nYrlIW8vjLvjYDRw61WwTc1eIv3iF+VYsvmqG9JGmMulyp/NLQchLwIIM+kN2xBljR1lcAVwzFz8zAMuD+dptsHXBikgWtg/5EYF3b92CSZW3k2JlD55IkjUmXPpXdeq9Kkg8xuMo4LMlmBqO4fg+4PMlZwDcYvEUSBkOCTwE2MRhS/Lr23duSXMDjb5p8W1VNd/7/Go8PKf50WyRJYzTb64R/Z5bjqqoumO3EVXXGTna9dKaTAWfv5DyrgFUzxDcAz5ktB0nSnjXblcp3Zog9BTgLeCYwa1GRJO1/Znud8Lum15M8DTiXwW2p1cC7dnacJGn/NWufSnui/T8Dr2HwsOLz2kOIkiQ9wWx9Kn8AvBK4GPipqnpoj2UlSZpIsw0p/g3ghxlM5njX0FQtD3aZpkWStP+ZrU9ll5+2lyTt3ywckqTeWFQkSb2xqEiSemNRkST1xqIiSeqNRUWS1BuLiiSpNxYVSVJvLCqSpN5YVCRJvbGoSJJ6Y1GRJPXGoiJJ6o1FRZLUG4uKJKk3YykqSb6e5KYkNyTZ0GKHJlmf5Pb2uaDFk+S9STYluTHJ84bOs6K1vz3JinH8FknS48Z5pfKSqlpSVUvb9nnAlVW1GLiybQOcDCxuy0rgIhgUIeB84AXAccD504VIkjQee9Ptr1OBS9v6pcBpQ/HLauBqYH6ShcBJwPqq2lZV9wLrgeV7OmlJ0uPGVVQK+Nsk1ydZ2WKHV9UWgPb5rBY/Arhz6NjNLbazuCRpTHb6jvoRO76q7kryLGB9kq/O0jYzxGqW+BNPMChcKwGOOuqoXc1VktTRWK5Uququ9nkP8HEGfSJ3t9tatM97WvPNwJFDhy8C7polPtP3XVxVS6tq6dTUVJ8/RZI0ZI8XlSRPSfK06XXgROBmYA0wPYJrBXBFW18DnNlGgS0D7m+3x9YBJyZZ0DroT2wxSdKYjOP21+HAx5NMf/9fVtXfJLkOuDzJWcA3gFe39muBU4BNwHeB1wFU1bYkFwDXtXZvq6pte+5nSJJ2tMeLSlXdAfzMDPFvAy+dIV7A2Ts51ypgVd85SpJ2z940pFiSNOEsKpKk3oxrSPFEeP5vXjbuFPZ51//BmeNOQVKPvFKRJPXGoiJJ6o1FRZLUG4uKJKk3FhVJUm8sKpKk3lhUJEm9sahIknpjUZEk9caiIknqjUVFktQbi4okqTcWFUlSbywqkqTeWFQkSb2xqEiSemNRkST1xqIiSerNxBeVJMuT3JZkU5Lzxp2PJO3PJrqoJJkHvA84GTgWOCPJsePNSpL2XxNdVIDjgE1VdUdVfR9YDZw65pwkab816UXlCODOoe3NLSZJGoMDx53ADygzxOoJjZKVwMq2+VCS20aa1XgdBnxr3El0lT9cMe4U9iYT9bcD4PyZ/gnutybq75df3+W/3Y90aTTpRWUzcOTQ9iLgrh0bVdXFwMV7KqlxSrKhqpaOOw/tOv92k82/38Ck3/66Dlic5JgkBwOnA2vGnJMk7bcm+kqlqh5Jcg6wDpgHrKqqjWNOS5L2WxNdVACqai2wdtx57EX2i9t8+yj/dpPNvx+Qqif0a0uStFsmvU9FkrQXsajsI5yuZnIlWZXkniQ3jzsX7ZokRyb5TJJbk2xMcu64cxo3b3/tA9p0Nf8b+AUGw6yvA86oqlvGmpg6SfJzwEPAZVX1nHHno+6SLAQWVtWXkjwNuB44bX/+t+eVyr7B6WomWFV9Dtg27jy066pqS1V9qa0/CNzKfj6rh0Vl3+B0NdKYJTkaeC5wzXgzGS+Lyr6h03Q1kkYjyVOBjwJvrKoHxp3POFlU9g2dpquR1L8kBzEoKB+sqo+NO59xs6jsG5yuRhqDJAE+ANxaVe8edz57A4vKPqCqHgGmp6u5Fbjc6WomR5IPAV8EfiLJ5iRnjTsndXY88FrghCQ3tOWUcSc1Tg4pliT1xisVSVJvLCqSpN5YVCRJvbGoSJJ6Y1GRJPXGoiL1KMn8JP9pD3zPi5O8aNTfI+0qi4rUr/lA56KSgd35d/hiwKKivY7PqUg9SjI9Q/RtwGeAnwYWAAcBv11VV7SJBz/d9r8QOA14GfAmBtPr3A48XFXnJJkC/hQ4qn3FG4FvAlcDjwJbgTdU1f/aE79PmotFRepRKxifqqrnJDkQ+KGqeiDJYQwKwWLgR4A7gBdV1dVJfhj4B+B5wIPA3wNfaUXlL4H3V9XnkxwFrKuqZyd5C/BQVf3hnv6N0mwOHHcC0j4swDvaS7geY/A6gsPbvn+qqqvb+nHAZ6tqG0CSvwJ+vO17GXDsYIopAJ7eXgYl7ZUsKtLovAaYAp5fVf8vydeBJ7d93xlqN9OrC6YdALywqv7vcHCoyEh7FTvqpX49CExfSTwDuKcVlJcwuO01k2uBn0+yoN0y+7dD+/6WwWShACRZMsP3SHsNi4rUo6r6NvCFJDcDS4ClSTYwuGr56k6O+SbwDgZvDPw74Bbg/rb719s5bkxyC/D6Fv8k8Io2K+6/HtkPknaRHfXSXiDJU6vqoXal8nFgVVV9fNx5SbvKKxVp7/CWJDcANwNfAz4x5nyk3eKViiSpN16pSJJ6Y1GRJPXGoiJJ6o1FRZLUG4uKJKk3FhVJUm/+P7RyK1qmT6eeAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.countplot(data.interest_level);\n",
    "pyplot.xlabel('target');\n",
    "pyplot.ylabel('Number of occurrences');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 从原始数据中分离输入特征x和输出y\n",
    "X = data.drop('interest_level', axis = 1)\n",
    "y = data['interest_level']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns = X.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 将数据分割训练数据与测试数据\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# 随机采样20%的数据构建测试样本，其余作为训练样本\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3,random_state = 0)\n",
    "#print('X_train.shape: %s %s'%X_train.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 各类样本不均衡，交叉验证是采用StratifiedKFold，在每折采样时各类样本按比例采样"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prepare cross validation\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 第一轮参数调整得到的n_estimators最优值（187），其余参数继续默认值\n",
    "用交叉验证评价模型性能时，用scoring参数定义评价指标。评价指标是越高越好，因此用一些损失函数当评价指标时，需要再加负号，如neg_log_loss，neg_mean_squared_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': range(3, 10, 2), 'min_child_weight': range(1, 6, 2)}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#max_depth 建议3-10， min_child_weight=1／sqrt(ratio_rare_event) =5.5\n",
    "max_depth = range(3,10,2)\n",
    "min_child_weight = range(1,6,2)\n",
    "param_test2_1 = dict(max_depth=max_depth, min_child_weight=min_child_weight)\n",
    "param_test2_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.60834, std: 0.00312, params: {'max_depth': 3, 'min_child_weight': 1},\n",
       "  mean: -0.60822, std: 0.00317, params: {'max_depth': 3, 'min_child_weight': 3},\n",
       "  mean: -0.60814, std: 0.00343, params: {'max_depth': 3, 'min_child_weight': 5},\n",
       "  mean: -0.59740, std: 0.00366, params: {'max_depth': 5, 'min_child_weight': 1},\n",
       "  mean: -0.59747, std: 0.00306, params: {'max_depth': 5, 'min_child_weight': 3},\n",
       "  mean: -0.59752, std: 0.00287, params: {'max_depth': 5, 'min_child_weight': 5},\n",
       "  mean: -0.60091, std: 0.00353, params: {'max_depth': 7, 'min_child_weight': 1},\n",
       "  mean: -0.60068, std: 0.00326, params: {'max_depth': 7, 'min_child_weight': 3},\n",
       "  mean: -0.59907, std: 0.00368, params: {'max_depth': 7, 'min_child_weight': 5},\n",
       "  mean: -0.61670, std: 0.00496, params: {'max_depth': 9, 'min_child_weight': 1},\n",
       "  mean: -0.60992, std: 0.00530, params: {'max_depth': 9, 'min_child_weight': 3},\n",
       "  mean: -0.60647, std: 0.00469, params: {'max_depth': 9, 'min_child_weight': 5}],\n",
       " {'max_depth': 5, 'min_child_weight': 1},\n",
       " -0.5973984751972113)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb2_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=187,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch2_1 = GridSearchCV(xgb2_1, param_grid = param_test2_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch2_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch2_1.grid_scores_, gsearch2_1.best_params_,     gsearch2_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([ 60.21695271,  62.21582732,  65.42835898, 106.44269643,\n",
       "        108.69151831, 113.07652154, 159.74890532, 158.15754352,\n",
       "        157.68308334, 204.77430778, 199.34357305, 153.85805473]),\n",
       " 'std_fit_time': array([ 0.61301546,  2.59306673,  0.29174315,  0.62393407,  2.36070153,\n",
       "         0.38455482,  1.01454341,  0.68867401,  0.47200448,  0.78742589,\n",
       "         1.55851767, 13.11738577]),\n",
       " 'mean_score_time': array([0.27736874, 0.27781224, 0.31068044, 0.44633918, 0.48713422,\n",
       "        0.45816536, 0.7963068 , 0.78389988, 0.77442493, 1.3223412 ,\n",
       "        1.24555798, 0.63985205]),\n",
       " 'std_score_time': array([0.02512569, 0.01291313, 0.03421232, 0.02462637, 0.026221  ,\n",
       "        0.028542  , 0.08154573, 0.08685349, 0.06329269, 0.14985465,\n",
       "        0.28159043, 0.04635347]),\n",
       " 'param_max_depth': masked_array(data=[3, 3, 3, 5, 5, 5, 7, 7, 7, 9, 9, 9],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_min_child_weight': masked_array(data=[1, 3, 5, 1, 3, 5, 1, 3, 5, 1, 3, 5],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'max_depth': 3, 'min_child_weight': 1},\n",
       "  {'max_depth': 3, 'min_child_weight': 3},\n",
       "  {'max_depth': 3, 'min_child_weight': 5},\n",
       "  {'max_depth': 5, 'min_child_weight': 1},\n",
       "  {'max_depth': 5, 'min_child_weight': 3},\n",
       "  {'max_depth': 5, 'min_child_weight': 5},\n",
       "  {'max_depth': 7, 'min_child_weight': 1},\n",
       "  {'max_depth': 7, 'min_child_weight': 3},\n",
       "  {'max_depth': 7, 'min_child_weight': 5},\n",
       "  {'max_depth': 9, 'min_child_weight': 1},\n",
       "  {'max_depth': 9, 'min_child_weight': 3},\n",
       "  {'max_depth': 9, 'min_child_weight': 5}],\n",
       " 'split0_test_score': array([-0.60826226, -0.60914928, -0.60937267, -0.60278146, -0.60199428,\n",
       "        -0.60126028, -0.60506111, -0.60338061, -0.60343898, -0.62415855,\n",
       "        -0.61648854, -0.61361001]),\n",
       " 'split1_test_score': array([-0.60553054, -0.60521148, -0.6047239 , -0.59416526, -0.59491966,\n",
       "        -0.59497586, -0.59617913, -0.59697648, -0.59487346, -0.61230491,\n",
       "        -0.60637231, -0.60275431]),\n",
       " 'split2_test_score': array([-0.60590906, -0.60556085, -0.60512549, -0.59355476, -0.59367239,\n",
       "        -0.5941703 , -0.60004119, -0.59651892, -0.59630422, -0.61023123,\n",
       "        -0.60169055, -0.60031833]),\n",
       " 'split3_test_score': array([-0.60781564, -0.60727119, -0.60731884, -0.59586417, -0.59699513,\n",
       "        -0.59670249, -0.59842534, -0.6023851 , -0.59719373, -0.61794796,\n",
       "        -0.6112038 , -0.60915972]),\n",
       " 'split4_test_score': array([-0.61420874, -0.61390126, -0.61414136, -0.60062688, -0.59976787,\n",
       "        -0.60049741, -0.60485875, -0.60412754, -0.60352152, -0.61885827,\n",
       "        -0.61384479, -0.60649515]),\n",
       " 'mean_test_score': array([-0.608345  , -0.60821859, -0.60813622, -0.59739848, -0.59746986,\n",
       "        -0.59752122, -0.60091297, -0.6006776 , -0.59906626, -0.61670021,\n",
       "        -0.60991997, -0.6064676 ]),\n",
       " 'std_test_score': array([0.00311513, 0.00316727, 0.00343401, 0.00366015, 0.00306276,\n",
       "        0.00287112, 0.00352521, 0.00325934, 0.00367927, 0.00496012,\n",
       "        0.00530112, 0.00468766]),\n",
       " 'rank_test_score': array([10,  9,  8,  1,  2,  3,  6,  5,  4, 12, 11,  7], dtype=int32),\n",
       " 'split0_train_score': array([-0.58137031, -0.58202842, -0.58312176, -0.50845671, -0.51527477,\n",
       "        -0.51996208, -0.39771691, -0.42359202, -0.43834517, -0.2763295 ,\n",
       "        -0.32737752, -0.35879776]),\n",
       " 'split1_train_score': array([-0.58169643, -0.58150329, -0.58283634, -0.50807877, -0.51589475,\n",
       "        -0.51972269, -0.39894487, -0.42366584, -0.4375018 , -0.27380813,\n",
       "        -0.32742413, -0.35777925]),\n",
       " 'split2_train_score': array([-0.58247785, -0.58355523, -0.58381303, -0.50901464, -0.51680894,\n",
       "        -0.5197773 , -0.40066759, -0.42489525, -0.44067539, -0.27877116,\n",
       "        -0.32635077, -0.35806625]),\n",
       " 'split3_train_score': array([-0.58161325, -0.58232714, -0.5827657 , -0.50946109, -0.51639168,\n",
       "        -0.52048944, -0.39857612, -0.42518637, -0.4393693 , -0.2784644 ,\n",
       "        -0.32612832, -0.3553493 ]),\n",
       " 'split4_train_score': array([-0.58087018, -0.58159686, -0.58236723, -0.50708036, -0.5134582 ,\n",
       "        -0.51763023, -0.39945696, -0.42400634, -0.43798374, -0.27229006,\n",
       "        -0.32360528, -0.3526273 ]),\n",
       " 'mean_train_score': array([-0.5816056 , -0.58220219, -0.58298081, -0.50841832, -0.51556567,\n",
       "        -0.51951635, -0.39907249, -0.42426917, -0.43877508, -0.27593265,\n",
       "        -0.3261772 , -0.35652397]),\n",
       " 'std_train_score': array([0.00052256, 0.00073911, 0.00048085, 0.00081852, 0.00117153,\n",
       "        0.00098119, 0.00097902, 0.0006519 , 0.00113107, 0.00254584,\n",
       "        0.00138873, 0.00226664])}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch2_1.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.597398 using {'max_depth': 5, 'min_child_weight': 1}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAELCAYAAADkyZC4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xl8FdX9+P/X+2YPO2EnILuCEENAxFoVVBaXohQXcGNRqVo/1vanxe1Tq1Wr8i2u1BYRxH2tiBYRRFQ+WssSA0IQDIsSUQIJexLI8v79MZNwc7lJbpbJDcn7+eA+uHfmnDPvmUDed87MnCOqijHGGFPbfOEOwBhjTMNkCcYYY4wnLMEYY4zxhCUYY4wxnrAEY4wxxhOWYIwxxnjCEowxxhhPeJpgRGS0iGwUkQwRubOcMpeLSLqIrBeRV/2WLxKRvSLyQUD55SKS5r52iMh8d/kwEdnnt+5PXu6bMcaYikV61bCIRAAzgRFAJrBSRBaoarpfmd7AXcAZqrpHRNr5NTEdiAd+49+uqp7pV/8d4D2/1ctV9aJa3xljjDFV5lmCAYYAGaq6BUBEXgcuBtL9ytwAzFTVPQCqmlWyQlWXisiw8hoXkWbAOcDk6gbYpk0b7datW3WrG2NMo7R69erdqtq2snJeJpjOwHa/z5nAaQFl+gCIyBdABPBnVV0UYvtjgaWqut9v2ekisgbYAdyuqusraqBbt26sWrUqxM0ZY4wBEJHvQynnZYKRIMsCBz6LBHoDw4BEYLmI9FfVvSG0PwGY7fc5FThBVQ+KyAXAfLftskGJTAWmAnTt2jWEzRhjjKkOLy/yZwJd/D4n4pxZBJZ5T1ULVHUrsJEgSSGQiCTgdMH9u2SZqu5X1YPu+4VAlIi0CayrqrNUdbCqDm7bttIzPGOMMdXkZYJZCfQWke4iEg2MBxYElJkPDAdwk0EfYEsIbV8GfKCq+SULRKSDiIj7fgjOvmXXeC+MMcZUi2ddZKpaKCK3AB/hXF+Zo6rrReQBYJWqLnDXjRSRdKAIuENVs8G5HRk4CWgqIpnAdar6kdv8eOCRgE1eCtwkIoVAHjBebS4CY2pVQUEBmZmZ5OfnV17YHPdiY2NJTEwkKiqqWvWlMf8OHjx4sNpFfmNCt3XrVpo1a0ZCQgJuh4FpoFSV7OxsDhw4QPfu3cusE5HVqjq4sjbsSX5jTMjy8/MtuTQSIkJCQkKNzlYtwRhjqsSSS+NR05+1JZhqKNyzh+w5cyk6eDDcoRhjTL1lCaYaDn76GVmPPUbGsOHsnD6dgp07wx2SMcbUO5ZgqqHl2Evo9vbbND3rLHLmvkDGueexY9qd5G/cFO7QjGnQ9u7dy9///vdq1X3iiSfIzc2t5Yhqx7Bhw6o9qsj8+fNJTz86AlcobeXn5zNkyBBOOeUUTj75ZO67775qbbsylmCqKa7/yXSe8Td6Ll5MqwkT2L94MVsvvpgfbpjKoa++ojHfnWeMVxpqgqmJwAQTipiYGD755BPWrFlDWloaixYt4quvvqr12LwcKqZRiE7sTId77qbtb29mz+uvk/PyK/wwaTKx/frResoUmo8ehUTaYTYNz/3vryd9x/7KC1ZBv07Nue9XJ5e7/s4772Tz5s0kJyczYsQI2rVrx5tvvsnhw4cZO3Ys999/P4cOHeLyyy8nMzOToqIi/vd//5edO3eyY8cOhg8fTps2bVi2bFnQ9ps2bcpvf/tbPv74Y1q1asXDDz/MH//4R3744QeeeOIJxowZw7Zt27jmmms4dOgQAM888wy/+MUvePfdd5k5cyZLlizh559/5uyzz+bzzz+nQ4cOx2wnLy+PyZMnk56eTt++fcnLyytdt3jxYu677z4OHz5Mz549mTt3Lk2bNqVbt25cccUVpbG/+uqrZGVlsWDBAj777DMefPBB3nnnHQDeeustbr75Zvbu3cvzzz/PmWeeWWb7IkLTpk0B59mmgoICT27esDOYWhLRsiVtbryRXks/psNfHqA4L48dt9/O5pGjyHnxRYrdf4zGmOp75JFH6NmzJ2lpaYwYMYLvvvuOFStWkJaWxurVq/n8889ZtGgRnTp1Ys2aNaxbt47Ro0dz66230qlTJ5YtW1ZucgE4dOgQw4YNY/Xq1TRr1ox7772XJUuW8O677/KnPzlTTLVr144lS5aQmprKG2+8wa233grA2LFj6dChAzNnzuSGG27g/vvvD5pcAJ599lni4+NZu3Yt99xzD6tXrwZg9+7dPPjgg3z88cekpqYyePBgZsyYUVqvefPmrFixgltuuYXbbruNX/ziF4wZM4bp06eTlpZGz549ASgsLGTFihU88cQT3H///QDs2LGDCy64oLStoqIikpOTadeuHSNGjOC00wLHIq45+2pdy3wxMbS67DJajhvHwU8/Jfv5Oex8+K/smvl3Wo0fT+urryLSxkAzDUBFZxp1YfHixSxevJiBAwcCcPDgQb777jvOPPNMbr/9dqZNm8ZFF110zLf3ikRHRzN69GgABgwYQExMDFFRUQwYMIBt27YBzjf+W265hbS0NCIiIti06ei116effpr+/fszdOhQJkyYUO52Pv/889LElJSURFJSEgBfffUV6enpnHHGGQAcOXKE008/vbReSZsTJkzg97//fbnt//rXvwZg0KBBpXF36tSJhQsXlpaJiIggLS2NvXv3MnbsWNatW0f//v1DOk6hsgTjEfH5aHbOOTQ75xzy0tLInjOX7FmzyJkzh+YXjyFhyhRievQId5jGHLdUlbvuuovf/OY3x6xbvXo1Cxcu5K677mLkyJGlZx+ViYqKKu0q8vl8xMTElL4vLCwE4PHHH6d9+/asWbOG4uJiYmNjS+v/+OOP+Hw+du7cSXFxMT5f+Z1EwbqkVJURI0bw2muvVVqnoi6tkrgjIiJK4y5Py5YtGTZsGIsWLar1BGNdZHUgLjmZxKeepOeiD2lx6Tj2v/8BWy64kO033UzuqlV2Q4AxIWrWrBkHDhwAYNSoUcyZM4eD7vNoP/74I1lZWezYsYP4+Hiuvvpqbr/9dlJTU4+pWxP79u2jY8eO+Hw+XnrpJYqKigCnW2ry5Mm8+uqr9O3bt0zXVqCzzjqLV155BYB169axdu1aAIYOHcoXX3xBRkYGALm5uWXOkN54443Sv0vObKqzX7t27WLvXmdWlLy8PD7++GNOOumkKrURCkswdSj6hBPoeN999Fr2CW1++1vyvv6a76++hm3jx7N/0Ueo+w/VGBNcQkICZ5xxBv3792fJkiVceeWVnH766QwYMIBLL72UAwcO8M033zBkyBCSk5N56KGHuPfeewGYOnUq559/PsOHD69RDDfffDPz5s1j6NChbNq0iSZNmgDw8MMPc+aZZ3LmmWcyY8YMZs+ezYYNG4K2cdNNN3Hw4EGSkpJ47LHHGDJkCABt27blhRdeYMKECSQlJTF06FC+/fbb0nqHDx/mtNNO48knn+Txxx8HYPz48UyfPp2BAweyefPmcuP2vwbz008/MXz4cJKSkjj11FMZMWIEF11U+7PN22CXYRzssjgvj33z55M99wUKfviBqK5daT1pIi3HjsUXFxe2uIwpz4YNG+jbt2+4w2iUSmbgbdPmmGmuPBXsZ26DXR4HfHFxtJowgZ4fLqTzk08S0aolOx/4CxnnnMuup5+hMCcn3CEaY0y12UX+ekAiImg+aiTNRo4gb/VqsufMZffMmWTPnk2LsZeQMHky0SecEO4wjWkwTjvtNA4fPlxm2UsvvcSAAQNqdTsfffQR06ZNK7Ose/fuvPvuu1Vuq+RusOOJdZHV0/lgDm/ZQs7cueyb/x5aWEiz884j4bopxCUnhzs004hZF1njY11kDVBMjx50/Mtf6PXJUhKmTuXQihVsGz+BbVddzYFPPkGLi8MdojHGVMjTBCMio0Vko4hkiMid5ZS5XETSRWS9iLzqt3yRiOwVkQ8Cyr8gIltFJM19JbvLRUSecre1VkRSvNy3uhLZti3tfn8bvT9ZSvu776bw55/JvPm3bLnwIva8+SbFAaf5xhhTX3iWYEQkApgJnA/0AyaISL+AMr2Bu4AzVPVk4Da/1dOBa8pp/g5VTXZfae6y84He7msq8Gyt7Uw94GvShNbXXkPPjxbR6W//D19cHD//6T4yzjmX3c8+S5F7T7sxxtQXXp7BDAEyVHWLqh4BXgcuDihzAzBTVfcAqGpWyQpVXQpU5emhi4EX1fEV0FJEOtZoD+ohiYykxYUX0u2dt+n6wgvEntyPXU8+xXfDz+HnBx/iSGZmuEM0xhjA2wTTGdju9znTXeavD9BHRL4Qka9EZHSIbT/kdoM9LiIxVdhegyEiNBl6Gl1nzaL7gvdoPmoUe954g80jR/HjH/5A3jfrwh2iMbWuoQ7XX9fzwYDzXM2AAQNITk5m8OBKr9dXi5cJJthAOYG3rEXidGkNAyYAs0WkZSXt3gWcBJwKtAZK7gEMZXuIyFQRWSUiq3bt2lXJpo4PsX360OmRv9Lr4yUkTJnMwc+Xs+2yy/j+2okc/OwzG4rGNBgNNcHURHXmgymxbNky0tLSqp3cKuPlczCZQBe/z4nAjiBlvlLVAmCriGzESTgry2tUVX9y3x4WkbnA7VXYHqo6C5gFzm3KIe/NcSCqfXva3X47CTfeyN433yLnxRfZ/psbiendi9aTp9DioguR6Ohwh2kaig/vhJ+/qd02OwyA8x8pd7XNB1M788HUFS/PYFYCvUWku4hEA+OBBQFl5gPDAUSkDU6X2ZaKGi25riLOUKKXACV9QQuAa927yYYC+/ySUaMS0bQpCVMm02vJYjo9+gj4Ivjp7rvJOG8Eu597jqL9tTtJlDF1xeaDqb35YESEkSNHMmjQIGbNmlWDn0r5PDuDUdVCEbkF+AiIAOao6noReQBYpaoL3HUjRSQdKMK5OywbQESW43SFNRWRTOA6Vf0IeEVE2uJ0iaUBN7qbXAhcAGQAucBkr/bteCFRUbS4+GKajxnDoS++JGfO8+z62wyy//FPWl52Ga0nXktUxwZ3H4SpKxWcadQFmw+mZvPBfPHFF3Tq1ImsrCxGjBjBSSedxFlnnRXScQqVp0PFqOpCnF/8/sv+5PdegT+4r8C6Qf9VqOo55SxX4Lc1ibehEhGa/vIMmv7yDPLT08meM5ecl14i5+WXaX7B+SRMmUKsB0N1G+Mlmw+mZvPBdOrUCXDOyMaOHcuKFStqPcHYk/yNTGy/fnT+f9PptWQxra+6ioMfL2XrJWP5Ycp1HPziC7shwNRrNh9M7cwHc+jQodI6hw4dYvHixbU+2RjYYJeNVlSnTrS/607a3HwTe954k5yXXmT7ddcT07cvCVMm03z0aCQqKtxhGlOG/3ww559/ful8MOBcoH/55ZfJyMjgjjvuwOfzERUVxbPPOs9cl8wH07Fjxwqvw1Tm5ptvZty4cbz11lsMHz486HwwycnJnHrqqVx44YVBx2676aabmDx5MklJSSQnJwedD6ZkMM4HH3yQPn36AEfngykuLi49yxk/fjw33HADTz31FG+//Xa5ce/YsYPrr7+ehQsXsnPnTsaOHQs4ifHKK68s7RqsTTbYZT0d7LKuFR85wv733yd7zlyObN5MZMeOtL72WlpedhkRTZuEOzxTT9hgl+Fj88GY45YvOpqW48bR4/0FJP7jWaITE8l69FEyhg8n629/o2BnVuWNGGOMH+siM2WIz0ezYcNoNmwYed98Q/bzc5zXC/No8atfkTB5EjG9e4c7TGNqxOaDqRvWRWZdZJU6sn07OS/MY+8776D5+TQ5+ywSplxH/JBTK7yTxTQ81kXW+FgXmfFUdJcudPjfe+m17BPa3Po/5H+zjh8mTmTbZZez/8MP0XJugzTGNG6WYEzIIlu1ou3NN9Prk6V0+POfKT5wgB9//wc2jz6fnJdeprgBjvNkjKk+SzCmynyxsbQafwU9PlxI4jNPE9m2LTsfeoiM4eeQ9eSTFO7eHe4QjTH1gCUYU23i89HsvPPo9tqrnPDqq8SdOpjsf/yTjHPO5ac/3cfhLVvDHaJpYBrqaMp1PVz/xo0bSU5OLn01b96cJ554olrbr4glGFMr4lMG0uWZZ+ix8N+0GDuWffPns+XCC9n+21vIdZ+kNqamGmqCqYnqDNd/4oknkpaWVjpIaHx8fOmDl7XJEoypVTHdu9Px/j87NwTcdCN5q1bx/ZVXsW38BPYvXoy6w2oYUx3+w/XfcccdTJ8+nVNPPZWkpCTuu+8+wBn65MILL+SUU06hf//+vPHGGzz11FOlw/UPHz683PabNm3KtGnTGDRoEOeddx4rVqxg2LBh9OjRgwULnMHgt23bxplnnklKSgopKSl8+eWXALz77rucd955qCo//fQTffr04eeffw66nby8PMaPH09SUhJXXHHFMcP1n3766aSkpHDZZZeVDoXTrVs3pk2bxpAhQxgyZAgZGRl8+eWXLFiwgDvuuIPk5GQ2b94MOMP1DxkyhD59+rB8+fIKj+nSpUvp2bMnJ5xwQog/hdDZczDGE5EJCbS99VYSrr+evf96l5wXXuDHW39H9Akn0HryJFpccgk+v0ECzfHn0RWP8m3Ot7Xa5kmtT2LakGnlrn/kkUdYt24daWlpLF68mLfffpsVK1agqowZM4bPP/+cXbt20alTJ/79738DzthhLVq0YMaMGSxbtqzCJ+FLhut/9NFHGTt2bOlw/enp6UycOJExY8aUDtcfGxvLd999x4QJE1i1ahVjx47lnXfeYebMmSxatCjk4frXrl1LSkoKUHa4/iZNmvDoo48yY8aM0sE6S4brf/HFF7ntttv44IMPGDNmDBdddBGXXnppafslw/UvXLiQ+++/n48//rjMUDH+Xn/99QpHfq4JSzDGU774eFpffRWtJoznwJIlZD8/h5//fD+7nnqaVlddSasrrySyVatwh2mOQzZcf82G6y9pf8GCBfz1r3+t9NhUhyUYUyckIoLmo0fTbNQo8latIvv5Oex++hmyn5tNy1+PpfWkSUR37RruME0VVHSmURdsuP6aDdcP8OGHH5KSkkL79u3LLVMTdg3G1CkRIf7UU+nyj2fp8cH7NL/wAva+9TabR59P5u9uI88dttyYYGy4/toZrr/Ea6+95ln3GFiCMWEU06sXnR56iJ5LPybhuus49OWXbLv8Cr6/+hoOfLIMLS4Od4imnvEfrn/JkiWlw/UPGDCASy+9lAMHDvDNN98wZMgQkpOTeeihh7j33nuBo8P1V3SRPxQ333wz8+bNY+jQoWzatCnocP0zZsxg9uzZbNiwIWgbN910EwcPHiQpKYnHHnss6HD9SUlJDB06lG+/PXqdq2S4/ieffJLHH38ccIbrnz59OgMHDiy9yB9M4JTJubm5LFmypLQ7zQuejkUmIqOBJ3GmTJ6tqsfMsSoilwN/BhRYo6pXussXAUOB/1PVi/zKvwIMBgqAFcBvVLVARIYB7wElD1/8S1UfqCg+G4usfik6eIh977xN9rx5FO74ieiePUmYPInmY8bgi44Od3gGG4ssnGy4/rIBRAAzgfOBfsAEEekXUKY3cBdwhqqeDNzmt3o6cE2Qpl8BTgIGAHHA9X7rlqtqsvuqMLmY+ieiaRNaT5xIr48+otP06Uh0ND/d+79knHsuu/85i6J9+8IdojGmCry8yD8EyFDVLQAi8jpwMeD/RNANwExV3QOgqqWTjqjqUvespAxVLb0NQkRWAImeRG/CRqKiaPGri2h+0YXkfvUV2c/PYdfjj7P7n/+k5aXjSJg4kajOncMdpjmO2XD9dcPLBNMZ2O73ORM4LaBMHwAR+QKnG+3PqroolMZFJArnDOd3fotPF5E1wA7gdlVdH6TeVGAqQFe7a6leExGanH46TU4/nfyNG8mZM4c9r77Gnldepfno0SRcN4XYfv0qb8iYAP/973/rZDujRo1i1KhRdbKt+sjLi/zB7qELvOATCfQGhgETgNki0jLE9v8OfK6qJY+ppgInqOopwNPA/GCVVHWWqg5W1cFt27YNcVMm3GJPPJFOjz5KryWLaX3ttRz89FO2/noc30+ezMHly2nM8xoZU195mWAygS5+nxNxziwCy7ynqgWquhXYiJNwKiQi9wFtgT+ULFPV/ap60H2/EIgSkbq9GmY8F9WxI+2n/ZFeny6j3R23c2TzFrbfMJWtF1/C3vnz0SNHwh2iMcblZYJZCfQWke4iEg2MBxYElJkPDAdwk0EfYEtFjYrI9cAoYIKqFvst7yDuk0ciMgRn37JraV9MPRPRrBkJ111Hr4+X0PGvfwVVfrrzLjJGjCT7+TkU1cLzDsaYmvEswahqIXAL8BGwAXhTVdeLyAMiMsYt9hGQLSLpwDLgDlXNBhCR5cBbwLkikikiJR2Z/wDaA/8RkTQRKXlE91JgnXsN5ilgvFq/SYMn0dG0HHsJ3Re8R5fnZhHdvTtZ06eTMfwcdj42nYJyBhs0xtQBVW20r0GDBqlpeHK/WaeZv/+Dpvc7WdNP7q8//nGa5n27MdxhNQjp6elh3f6ePXt05syZ1ar7+OOP66FDh2o5otpx9tln68qVK6tV991339X169dXua0nnnhCTz75ZO3Xr58+/vjj5ZYL9jMHVmkIv2PtSX7T4MT1P5nOM/5Gz48+otWECexfsoStF1/MDzdM5dB//mM3BBzHbD6YY1VnPph169bx3HPPsWLFCtasWcMHH3zAd999V+uxWYIxDVZ0Ymc63HM3vT9ZStvbbiN/wwZ+mDyFbeMuZd8H/0YrGATQ1E82H0ztzAezYcMGhg4dSnx8PJGRkZx99tnVejanMjaasmnwIlq2pM2Nv6H15EnsW7CAnDlz2XH77eyaMYPWkybSctw4fO54UiZ0Pz/8MIc31O58MDF9T6LD3XeXu97mg6md+WD69+/PPffcQ3Z2NnFxcSxcuJDBgysd+aXKLMGYRsMXE0Oryy6j5bhxHPz0M7LnPM/Oh//Krmdm0mr8eFpdfRVR7dqFO0wTIpsPpvrzwfTt25dp06YxYsQImjZtyimnnEJkZO2nA0swptERn49m5wyn2TnDyVuzhuzn55D93HPkzJ1L8zG/ImHKFGJ69gx3mPVeRWcadUFtPphy2w5lPpjrrruO6667DoC7776bxMTaH3XLrsGYRi3ulFNIfOpJei76kBaXjmP/B/9my4UXsf3Gm8hdudJuCKhnbD6Y2psPJivLGfrxhx9+4F//+pcn88LYGYwxQPQJJ9Dxvvto+z//44539grfX3MtsUlJJEyZQrMR5yEREeEOs9Hznw/m/PPPL50PBpwL9C+//DIZGRnccccd+Hw+oqKiePbZZ4Gj88F07NiRZcuWVTuGm2++mXHjxvHWW28xfPjwoPPBJCcnc+qpp3LhhRcGnd7gpptuYvLkySQlJZGcnBx0PpiSwTgffPBB+vTpAxydD6a4uLj0LGf8+PHccMMNPPXUU7z99tvlxu1/DQZg3LhxZGdnExUVxcyZM2nlwdTlns4HU9/ZfDCmPMV5eeybP5/suS9Q8MMPRHXp4twQ8Otf44uLC3d4YWPzwYSPzQdjTAPhi4uj1YQJ9PxwIZ2fepLI1q3Z+ZcHyRh+DrueeprCnJxwh2hMvWddZMZUQCIiaD5yJM1GjCAvNZXs5+ew++9/J/v552kx9hISJk0iulu3cIdpqsjmg6kb1kVmXWSmig5v2UrO3Lnse+89tKCAZuedS+spU4h3b5dtyKyLrPGxLjJj6lBMj+50/MsD9PpkKQm/mcqhFSv5fsKVbLvyKg4sXYoWF1feyHGsMX8pbWxq+rO2BGNMNUW2aUO7226j9ydLaX/PPRTu3Enmb29hywUXsueNNykO6IJpCGJjY8nOzrYk0wioKtnZ2WWe86kq6yKzLjJTS7SwkAOLF5P9/Bzy168nIiGB1ldfRcvx44n04BbQcCgoKCAzM5P8/Pxwh2LqQGxsLImJiURFRZVZHmoXmSUYSzCmlqkquf9dQfac5zn0+XIkLo6W48bRetJEoj14WtqYumYJJgSWYIzX8jdtImfuC+z74AMoKqLZqJEkTLmOuAH9wx2aMdVWLy7yi8hoEdkoIhkicmc5ZS4XkXQRWS8ir/otXyQie0Xkg4Dy3UXkvyLynYi84U7HjIjEuJ8z3PXdvNw3Y0IR26cPnf76ML0+XkLClMkcWv5/bLvsMr6/diIHP/uswd8QYBo3zxKMiEQAM4HzgX7ABBHpF1CmN3AXcIaqngzc5rd6OnBNkKYfBR5X1d7AHuA6d/l1wB5V7QU87pYzpl6Iat+edrffTq9Pl9Fu2jSO/PAD239zI1vGjGHvO/+i+MiRcIdoTK3z8gxmCJChqltU9QjwOnBxQJkbgJmqugdAVbNKVqjqUqDMCG7iDB96DlAy4M484BL3/cXuZ9z150pFw40aEwYRTZuSMHkSvZYsptNjjyIRkfx0zz1sPvc8dj/3HEX794c7RGNqjZcJpjOw3e9zprvMXx+gj4h8ISJficjoStpMAPaqasn40/5tlm7PXb/PLW9MvSNRUbQYM4bu89+ly+zZxPTuxa6/zSBj2HB2PvIoBTt2hDtEY2rMy6Figp09BN5REAn0BoYBicByEemvqnur0WYo20NEpgJTAbp27VrOZoypGyJC01+eQdNfnkH+hg1kz5lLzksvkfPSS8T260d8SgpxKSnEpwwksm3bcIdrTJV4mWAygS5+nxOBwK9lmcBXqloAbBWRjTgJZ2U5be4GWopIpHuW4t9myfYyRSQSaAEcMyKhqs4CZoFzF1l1dswYL8T27Uvn6Y/R7ve3sfftt8lduYo9r79Ozjyn5zeqa1c34QwkftAgonv0qHDSKWPCzcsEsxLoLSLdgR+B8cCVAWXmAxOAF0SkDU6X2ZbyGlRVFZFlwKU413QmAu+5qxe4n//jrv9EG/M92Oa4FdWpE23d6XT1yBHy09PJTf2avK9TOfj55+ybPx+AiBYtiPNLOLEnn4zPncnQmPrA0+dgROQC4AkgApijqg+JyAPAKlVd4F6E/xswGigCHlLV1926y4GTgKZANnCdqn4kIj1wkktr4GvgalU9LCKxwEvAQJwzl/GqWm6yAnvX2db3AAAgAElEQVQOxhx/VJWC778nd3UquV+nkrc6lSNbtwLOdZ3YAQOITxlIXMog4gYmN5gRBEz9Yg9ahsASjGkICnNyyPv6a3JTnYSTt349FBQAEN2zp5NwBqYQPyiFqK5drVvN1JglmBBYgjENUXF+Pvnr1jndaqtXk5uWRvG+fQBEtGlD/MCBzo0Dg1KI7dsXCRhnypjKWIIJgSUY0xhocTFHNm92Ek7qanJTv6Zgu/MEgcTGEpeUVHodJy45mYhmzcIcsanvLMGEwBKMaawKsrLIS/2a3NTV5KV+Tf6GDVBUBCLE9OnjJJyUQc7t0Z06WbeaKcMSTAgswRjjKD50iLxvvjl6HSctjeJDhwCI7NChzHWcmBNPRCIiwhyxCadQE4yXtykbY44TviZNaDJ0KE2GDgVAi4o4vGlTacLJTU1l/8IPnbLx8cQlJ5dex4lLSsLXpEk4wzf1VKVnMCLSE8h0bwUeBiQBL1bwtP1xw85gjAldwY4dZa7jHN64EVQhIoLYk046mnAGphDVvl24wzUeqrUuMhFJAwYD3YCPcB5oPFFVL6iFOMPKEowx1Vd04AB5aWtKr+PkrV2L5uUBEJWYWHodJy5lIDG9eiE+m6G9oajNLrJiVS0UkbHAE6r6tIh8XfMQjTHHs4hmzWh65i9peuYvAdCCAvK//Za81FRyV6dy6D//Yf+C9wHwNW9O3MBk4t3rOLEDBuCrwVzv5vgQSoIpEJEJOMOw/MpdZjfOG2PKkKgo4gYMIG7AAFpPnOiMOrB9+9HrOF+nsuuzz53CUVHE9et3tFstJYXI1q3DuwOm1oXSRdYPuBH4j6q+5o4tdoWqPlIXAXrJusiMqVuFe/aQl5bmnOWkfk3+2rVoyagD3bqVuY4T3b2b3R5dT3lym7KItAK6qOramgRXX1iCMSa8io8cIX/d+tIbB/JSUyna69w/FNGqlTtVgTOgZ9zJJyPR0WGO2EAtXoMRkU+BMW7ZNGCXiHymqn+ocZTHqU83ZvHA++mIQIRP8IkgIvgEfCV/u8t9Qum6csuKW9ZXUtYtH1jW51dWAsr6jm1XRNxtelA2SLz+sfl8AWVFkHL2w+e/3ldSnoB9D1I2IGZz/PFFRxOfMpD4lIEk4AzmeWTrVnJXry59EPTg0qUASEwMsQP6l944ED9wIBEtWoR3B0yFQrkG00JV94vI9cBcVb1PRBrEGUx1NYuN4uTOLSguVoq15OX85yhWKHKXq3J0fTEUFhWHVFYVivzqlZZVLX3vrAsoG9BuYxOYrCtKRpWWDZIQnWR5bNlgCdNJ2EHKBnz58P/CcfQLiLscp6zgxOwsB8F5LyVl/Ne5idZXYfmjx4BgbSBQJoaKtwmBMZdfXtz2S2Lwb6N0WVRrfKePRH4x0imfk4Okr0XXrSVv3Vrynn8eZhUBIN174EtKJiIpmcikU/B17ozP5wvY77LbLLuPZZeVW96+wFRLKAkmUkQ6ApcD93gcz3Fh0AmtGHRC/R8GvUwyKklcxccmo5JkFWrZouKA5OmuLyoOUrYkKRZTtmyZdv3X+8cRrN3yYq56WWdfgpQNEq+zL8fGW1SsHCkKPE7B2/U/poHHr7j42HgVSsup+/Ms+znM/8DqVCSQAn1SiOlxhD57t3Ny9lb65Wyj778X0vS9f3EEyI5tTnrrbqxP6E56625sbtGJYl/tjTpQbpIKSJolXwrKJHPKJvEy5ctpozQx+yVsX0B5SuMJVr78Ni4Y0JHLB3epaHdrLJQE8wDO8y9fqOpKdz6W7zyNytSKkm/cEUFnkzYNQWDSKUle+L33T0jByuP8KVu+3Dacz8HKO8srKF9cdpvFFZUPmlj9yw8pjW97UTHR278nbuM3xG1az2mb1nPmN04nS1FMLId6nMTB3v040LsfB7qfRGFcfOmXgbL7eLT9wG0WuwdQg5RHK2gjoDy4vRJUVN7/uB1bHo5+mam0fDEUUXxMeVTJO1Lk4b9Mh41FZhf5jWlwCn7+ufROtdzU1Rz+dqPz29bnI+bEE8tMPR3VoUO4wz3u1OaT/InA08AZOF9E/g/4napm1kag4WQJxpjGoejgIfLWpB0dQXrNWjQ3F4DITh2P3jiQkkJM7942mGclajPBLAFexZmOGOBq4CpVHRFCEKOBJ3GmTJ4d7NkZEbkc+DNO8lqjqle6yycC97rFHlTVeSLSDFjuVz0ReFlVbxORScB04Ed33TOqOrui+CzBGNM4aWEh+d9udM9yUslLTaUwKwsAX9OmxCUnlz6PE5c0AF98fJgjrl9qdSwyVU2ubFmQehHAJmAEkAmsBCaoarpfmd7Am8A5qrpHRNqpapaItAZW4YyBpsBqYJCq7gncSeD3qvq5m2AGq+otle10CUswxhhwrk8U/Pjj0YSzOpXDGRnOxaDISGL79nW71VKcOXLatg13yGFVm2OR7RaRq4HX3M8TgOwQ6g0BMlR1ixvQ68DFQLpfmRuAmSWJQ1Wz3OWjgCWqmuPWXQKM9ouhJDm1o+wZjTHGVJmIEJ2YSHRiIi3GjAGgaN8+8tLSSqee3vP66+TMmwdAVNeuZa7jRHfvboN5BhFKgpkCPAM8jnM28SUwOYR6nYHtfp8zgdMCyvQBEJEvcLrR/qyqi8qp2zmg7gTgDS17CjZORM7COXP6vapuxxhjqiGiRQuann02Tc8+GwA9coT89PTSGwcOfv45++bPLy0bV5JwUlKI7d8fX0xMOMOvFypNMKr6A86T/KVE5DbgiUqqBrs3NrA/LhLoDQzDuZ6yXET6h1h3PHCN3+f3gdfceWtuBOYB5xwTlMhUYCpA165dK9kFY4xxSHS0M9FacjIJUyY7ow5s21Zm6umDy5Y5ZaOiiO3fv3Qgz7iBA4lsVf+fnatt1Z3R8g9UnmAyAf+neBKBHUHKfKWqBcBWEdmIk3AycZKOf91PSz6IyClApKquLlmmqv7dds8BjwYLSlVnAbPAuQZTyT4YY0xQIkJM9+7EdO9Oy3G/BqAwJ4e8r78md7Vz40D2vBdh9vMARPfoUXrjQPygFKK6dm3wIwRUN8GEclRWAr3d0Zd/xDnjuDKgzHycrq4XRKQNTpfZFmAz8LA7uCbASOAuv3oT8LseAyAiHVX1J/fjGGBD6LtjjDE1F9m6Nc3OPZdm554LQHF+Pvnr1pUmnP0fLWbvW28DEJGQcPTGgUEpxPbti0Q1rJlQqptgKv3m705SdgvOKAARwBxVXS8iDwCrVHWBu26kiKQDRcAdJWciIvIXnCQF8EDJBX/X5UDgjJq3isgYoBDIASZVc9+MMaZW+GJjiR88mPjBzg1XWlzMkc2by0w9fWDJEgAkNpa4pKTSGwfikpOJaNYsnOHXWLm3KYvIAYInEgHiVLW6yanesNuUjTHhVpCVVeY6Tv6GDVBUBCLE9OlTOvV0fMpAIjt1qhfdap7MB9PQWIIxxtQ3xYcOkffNN6VTFuSlpVF86BAAkR06EJ8ysPQ6TsyJJ4Zl1IHafA7GGGNMHfE1aUKToUNpMnQoAFpUxOFNm0qv4+SmprJ/4YdO2fh45862kplAk5LwNWkSzvDLsDMYO4MxxhxnCnbsKHMd5/DGjc6oAxERxJ50Upmpp6Pat6v17VsXWQgswRhjGoKiAwfIS1tTeh0nb80aND8fgKjExNLrOHEpA4np1avGow5YggmBJRhjTEOkBQXkf/ut39TTqRTt3g2Ar3lz4gYm03Lsr2k+elS12q+1azDl3E22D2cwyv+vZKwxY4wx9YNERRE3YABxAwbApEnOYJ7bt5e5jlOQ6f1IWqFc5J+B8wT+qzi3KI8HOgAbgTmUfeLeGGNMPSMiRHftSnTXrrQcewlwdEZPL4XSETdaVf+pqgdUdb871MoFqvoG0PgG1zHGmAagLp6nCSXBFIvI5SLic1+X+61rvBdwjDHGVCiUBHMVzqjFWe7rGuBqEYkDQp7cyxhjTOMSynD9W4BflbP6/2o3HGOMMQ1FpWcwIpIoIu+KSJaI7BSRd0QksS6CM8YYc/wKpYtsLrAA6IQzq+T77jJjjDGmXKEkmLaqOldVC93XC0Bbj+MyxhhznAslwewWkatFJMJ9XQ1kV1rLGGNMoxZKgpmCM8HXz8BPwKXAZC+DMsYYc/yrNMGo6g+qOkZV26pqO1W9BPh1KI2LyGgR2SgiGSJyZzllLheRdBFZLyKv+i2fKCLfua+Jfss/ddtMc1/t3OUxIvKGu63/iki3UGI0xhjjjerOB/MH4ImKCohIBDATGAFkAitFZIGqpvuV6Q3cBZyhqnv8kkVr4D5gMM7DnKvdunvcqlepauAoldcBe1S1l4iMBx4Frqjm/hljjKmh6o7ZHMoYA0OADFXdoqpHgNeBiwPK3ADMLEkcqprlLh8FLFHVHHfdEmB0Jdu7GJjnvn8bOFfqw9yixhjTSFU3wYQyRExnwH+4zkx3mb8+QB8R+UJEvhKR0SHWnet2j/2vXxIpraOqhTgjPieEtDfGGGNqXbldZOUM0w/O2UtcCG0HO3sIbC8S6I0zInMisFxE+ldS9ypV/VFEmgHv4Axd82KI20NEpgJTAbp27Vr5XhhjjKmWcs9gVLWZqjYP8mqmqqFcu8kEuvh9TsQZ9j+wzHuqWqCqW3GmAOhdUV1V/dH9+wDOFAJDArcnIpFACyAnyH7NUtXBqjq4bVt7nMcYY7xSs3kzK7YS6C0i3UUkGmcemQUBZeYDwwFEpA1Ol9kW4CNgpIi0EpFWwEjgIxGJdMshIlHARcA6t60FQMndZpcCn2hjnq7TGGPCrLp3kVVKVQtF5BacZBEBzFHV9SLyALBKVRdwNJGkA0XAHaqaDSAif8FJUgAPqGqOiDTBSTRRbpsfA8+5ZZ4HXhKRDJwzl/Fe7ZsxxpjKSWP+kj948GBdtSrwbmdjjDEVEZHVqjq4snJedpEZY4xpxCzBGGOM8YQlGGOMMZ6wBGOMMcYTlmCMMcZ4whKMMcYYT1iCMcYY4wlLMMYYYzxhCcYYY4wnLMEYY4zxhCUYY4wxnrAEY4wxxhOWYIwxxnjCEowxxhhPWIIxxhjjCUswxhhjPGEJxhhjjCc8TTAiMlpENopIhojcWU6Zy0UkXUTWi8irfssnish37muiuyxeRP4tIt+65R/xKz9JRHaJSJr7ut7LfTPGGFOxSK8aFpEIYCYwAsgEVorIAlVN9yvTG7gLOENV94hIO3d5a+A+YDCgwGoRWQAcBv6fqi4TkWhgqYicr6ofuk2+oaq3eLVPxhhjQuflGcwQIENVt6jqEeB14OKAMjcAM1V1D4CqZrnLRwFLVDXHXbcEGK2quaq6zC17BEgFEj3cB2OMMdXkZYLpDGz3+5zpLvPXB+gjIl+IyFciMjrUuiLSEvgVsNRv8TgRWSsib4tIl9rYCWOMMdXjZYKRIMs04HMk0BsYBkwAZruJo8K6IhIJvAY8papb3MXvA91UNQn4GJgXNCiRqSKySkRW7dq1qwq7Y4wxpiq8TDCZgP9ZRCKwI0iZ91S1QFW3AhtxEk5ldWcB36nqEyULVDVbVQ+7H58DBgULSlVnqepgVR3ctm3bauyWMcaYUHiZYFYCvUWku3tBfjywIKDMfGA4gIi0weky2wJ8BIwUkVYi0goY6S5DRB4EWgC3+TckIh39Po4BNtT6HhljjAmZZ3eRqWqhiNyCkxgigDmqul5EHgBWqeoCjiaSdKAIuENVswFE5C84SQrgAVXNEZFE4B7gWyBVRACeUdXZwK0iMgYoBHKASV7tmzHGmMqJauBlkcZj8ODBumrVqnCHYYwxxxURWa2qgysrZ0/yG2OM8YQlGGOMMZ6wBGOMMcYTlmCMMcZ4whKMMcYYT1iCMcYY4wlLMMYYYzxhCcYYY4wnLMEYY4zxhCUYY4wxnvBsLLKGbOXPK/nn2n8SKZFE+CKIkAgifZFESMSxn933/utDqhekXHXrlS533/vEvlcYY7xnCaYairWYI0VHyCvOo1ALKSouokiLKCwupEiLKCouKl0ebH2xFoc1fp/4giepShJYlZNkkHVRvqiaJUmJrHa9CJ8lV2PqkiWYajit42mc1vG0atcv1uLSRBSYmMpLVCXLy13n9768xFayvLA4+LoKt+33vqCogDzNK7edimIp0qJa/ElUnSDHJMtQkmtgoipZVpXk6v85yhdV4TYifZEkxCXQPr49bePaEhURFdbjZkx1WIIJA5/48ImPKF/j+6WhqqWJprKEV9nZYXlni1WtV2kS9k+uxQXkFeWFnKD926luchWE1rGtad+kPe3j3VfA+3bx7YiLjKvln5YxNWMJxtQpEXG+oRPpzBLUiARLruUlpoKiAnbn7WZn7k7ndcj5O/NgJqlZqew7vO+Y9ptHNy+beAISUbsm7WgW1Qx3HiVjPGcJxpg6UtXkeiInlrsurzCPrNys0sTjn4R25u7k25xvyc7LRik731NcZNyxZ0B+Z0Ht49vTKraVXasytcISjDHHobjIOE5ofgInND+h3DIFRQXsytt1bAJy/17580p25e6iUAvL1IvyRZUmmzLJyC8JtYlrQ6TPfn2Yinn6L0RERgNP4nxfm62qjwQpcznwZ0CBNap6pbt8InCvW+xBVZ3nLh8EvADEAQuB36mqikhr4A2gG7ANuFxV93i1b8bUd1ERUXRq2olOTTuVW6aouIic/JygZ0FZuVmsz17PJ9s/4XDR4TL1fOKjTVybY5JQaWJyk1FMRIzXu2nqMc+mTBaRCGATMALIBFYCE1Q13a9Mb+BN4BxV3SMi7VQ1y00Wq4DBOIlnNTDILbMC+B3wFU6CeUpVPxSRx4AcVX1ERO4EWqnqtIpitCmTjamcqrLv8L6jSchNRFm5WWWS0sGCg8fUbRXTqux1oPh2Zc6I2se3p0lUkzDslamJUKdM9vIMZgiQoapb3IBeBy4G0v3K3ADMLDnTUNUsd/koYImq5rh1lwCjReRToLmq/sdd/iJwCfCh2/Ywt/484FOgwgRjjKmciNAytiUtY1tyYuvyrwsdKjh0TFdcaRLK3cnaXWvZc/jYToWmUU2P6YILvEbUIqaF3ZxwHPIywXQGtvt9zgQCHx7pAyAiX+B0o/1ZVReVU7ez+8oMshygvar+BKCqP4lIu2BBichUYCpA165dq75XxpigmkQ1oUeLHvRo0aPcMoeLDh9zc4L/54w9GezO333Mw8gxETHHJqGARNQ6tjURvkZ2a2I952WCCfZ1I7A/LhLojXPmkQgsF5H+FdQNpc0KqeosYBY4XWRVqWuMqZmYiBi6NOtCl2Zdyi1TWFx49BbtIF1xaVlp7MzdSWFx2ZsTIiWStvFtg58Fue/todW65WWCyQT8/xUlAjuClPlKVQuArSKyESfhZHK0u6uk7qfu8sRy2twpIh3ds5eOQBbGmONOpC+SDk060KFJB2gbvEyxFrMnf0/Qrridh3ayac8mlv+4nLzCvDL17KHVuuVlglkJ9BaR7sCPwHjgyoAy84EJwAsi0gany2wLsBl4WERaueVGAnepao6IHBCRocB/gWuBp90yC4CJwCPu3+95tmfGmLDyiY+EuAQS4hLol9AvaBlV5UDBgaBnQT/n/kzmwUxW71zN/iP7j6lrD63WDs8SjKoWisgtwEc411fmqOp6EXkAWKWqC9x1I0UkHSgC7lDVbAAR+QtOkgJ4oOSCP3ATR29T/tB9gZNY3hSR64AfgMu82jdjTP0nIjSPbk7z6Ob0btW73HK5Bblk5WaVOQv6+dDPpZ83ZG8gOz/7mHr20GrlPLtN+XhgtykbY0JR5qHVckZP2JW765jx5sp7aNX/du3j8aHV+nCbsjHGNAhVfmj1kNMN5981ty57XfkPrca2CTpiwvH+0KolGGOMqQURvgjaxrelbXxb+rfpH7RMeQ+tlvy9Zd8Wvvrpqwbz0KolGGOMqSOhPrR68MhBsnKzjp4F+XfLHUcPrVqCMcaYeqZpdFOaRjelR8vyH1rNL8xnV+6uY7riSh5ezdiTwa68XceMqF3y0OoVJ17BtSdf6+l+WIIxxpjjUGxkLF2ad6FL8/IfWi0oLiA7LzvoED6tYluVW6+2WIIxxpgGKsoXVelDq15qvDdoG2OM8ZQlGGOMMZ6wBGOMMcYTlmCMMcZ4whKMMcYYT1iCMcYY4wlLMMYYYzxhCcYYY4wnLMEYY4zxhCUYY4wxnvA0wYjIaBHZKCIZInJnkPWTRGSXiKS5r+v91j0qIuvc1xV+y5f7ld8hIvPd5cNEZJ/fuj95uW/GGGMq5tlYZCISAcwERgCZwEoRWaCq6QFF31DVWwLqXgikAMlADPCZiHyoqvtV9Uy/cu8A7/lVXa6qF3mwO8YYY6rIyzOYIUCGqm5R1SPA68DFIdbtB3ymqoWqeghYA4z2LyAizYBzgPm1GLMxxpha4uVoyp2B7X6fM4HTgpQbJyJnAZuA36vqdpyEcp+IzADigeFA4JnPWGCpqu73W3a6iKwBdgC3q+r62tmVRkTVeeH+rcVH3+N+9l9fZhmh1TlmvVbQZkVxUI06Je+p4r5VN86SZVQ/zpL5PI7Ztv/fgceivDIVtFPpNgLLBNtmqHEEWVbtOIK1U904ArcRSplQtlFRO1XZRjllq3rMAM64DUbcj5e8TDDBpkzTgM/vA6+p6mERuRGYB5yjqotF5FTgS2AX8B+gMKDuBGC23+dU4ARVPSgiF+Cc2fQ+JiiRqcBUgK5du1Z9rwC2LofPHnX3SKn6L1SqUSfYP5wq1in3l1/AP0JzHBEQofS/W8n78v4OWsa/HSquX2EZahiHfzs1iaNkGwI+X8XbqjTWgGNTrTgqO+ZVjSPIuurE0XUoXvMywWQC/jPhJOKcWZRS1Wy/j88Bj/qtewh4CEBEXgW+K1knIgk4XXBj/crv93u/UET+LiJtVHV3wDZnAbMABg8eXM3fqArFhTg/KB/4IkoCcz4H/nCPWeYr+w+j3PXl1acadYL9w6usjq/sfwQv9i1oHFWNs7r7VtHxDOXnUsk2y42Dqu1bsF8OdTTlrTE14WWCWQn0FpHuwI/AeOBK/wIi0lFVf3I/jgE2uMsjgJaqmi0iSUASsNiv6mXAB6qa79dWB2CnqqqIDMG5vuSfwGpP97OclzHGmHJ5lmBUtVBEbgE+AiKAOaq6XkQeAFap6gLgVhEZg9P9lQNMcqtHAcvF+Za2H7haVf27yMYDjwRs8lLgJhEpBPKA8apqfT7GGBMm0ph/Bw8ePFhXrVoV7jCMMea4IiKrVXVwZeXsSX5jjDGesARjjDHGE5ZgjDHGeMISjDHGGE9YgjHGGOMJSzDGGGM80ahvUxaRXcD31azeBthdaam6V1/jgvobm8VVNRZX1TTEuE5Q1baVFWrUCaYmRGRVKPeB17X6GhfU39gsrqqxuKqmMcdlXWTGGGM8YQnGGGOMJyzBVN+scAdQjvoaF9Tf2CyuqrG4qqbRxmXXYIwxxnjCzmCMMcZ4whJMJURkjohkici6ctaLiDwlIhkislZEUupJXMNEZJ+IpLmvP9VBTF1EZJmIbBCR9SLyuyBl6vx4hRhXOI5XrIisEJE1blz3BykTIyJvuMfrvyLSrZ7ENUlEdvkdr+u9jstv2xEi8rWIfBBkXZ0frxDjCufx2iYi37jbPWb4eE//T6qqvSp4AWcBKcC6ctZfAHyIM+3gUOC/9SSuYTiTstXlseoIpLjvmwGbgH7hPl4hxhWO4yVAU/d9FPBfYGhAmZuBf7jvxwNv1JO4JgHP1OXx8tv2H4BXg/28wnG8QowrnMdrG9CmgvWe/Z+0M5hKqOrnOJOhledi4EV1fAW0FJGO9SCuOqeqP6lqqvv+AM4MpZ0DitX58QoxrjrnHoOD7sco9xV4UfRiYJ77/m3gXBFv50sOMa6wEJFE4EJgdjlF6vx4hRhXfebZ/0lLMDXXGdju9zmTevDLy3W6283xoYicXJcbdrsmBuJ8+/UX1uNVQVwQhuPldqukAVnAElUt93ipM6vrPiChHsQFMM7tUnlbRLp4HZPrCeCPQHE568NyvEKIC8JzvMD5crBYRFaLyNQg6z37P2kJpuaCfTuqD9/2UnGGczgFeBqYX1cbFpGmwDvAbaq6P3B1kCp1crwqiSssx0tVi1Q1GUgEhohI/4AiYTleIcT1PtBNVZOAjzl61uAZEbkIyFLV1RUVC7LM0+MVYlx1frz8nKGqKcD5wG9F5KyA9Z4dM0swNZcJ+H8bSQR2hCmWUqq6v6SbQ1UXAlEi0sbr7YpIFM4v8VdU9V9BioTleFUWV7iOl9/29wKfAqMDVpUeLxGJBFpQh12j5cWlqtmqetj9+BwwqA7COQMYIyLbgNeBc0Tk5YAy4ThelcYVpuNVsu0d7t9ZwLvAkIAinv2ftARTcwuAa907MYYC+1T1p3AHJSIdSvqeRWQIzs862+NtCvA8sEFVZ5RTrM6PVyhxhel4tRWRlu77OOA84NuAYguAie77S4FP1L0yG864Avrox+Bc1/KUqt6lqomq2g3nAv4nqnp1QLE6P16hxBWO4+Vut4mINCt5D4wEAu889ez/ZGRtNNKQichrOHcYtRGRTOA+nIueqOo/gIU4d2FkALnA5HoS16XATSJSCOQB473+j4bzTe4a4Bu3/x7gbqCrX1zhOF6hxBWO49URmCciETgJ7U1V/UBEHgBWqeoCnMT4kohk4HwTH+9xTKHGdauIjAEK3bgm1UFcQdWD4xVKXOE6Xu2Bd93vTpHAq6q6SERuBO//T9qT/MYYYzxhXWTGGGM8YQnGGGOMJyzBGGOM8YQlGGOMMZ6wBGOMMcYTlmCMMcZ4whKMMccBd8j1ao0s4A4V36k22jKmKizBGNPwTQI6VVbImNpmCcaYKhCRbiLyrYjMFpF1IvKKiJwnIl+IyHciMsR9fSnO5FNfisiJbt0/iMgc9/0At358OdtJEJHFbhv/xG9AQhG5WpwJwdJE5J/uE/eIyOHwOT4AAAI7SURBVEER+ZuIpIrIUnfIl0uBwcArbvk4t5n/cct9IyIneXnMTONlCcaYqusFPAkkAScBVwK/BG7//9u7e9cogjiM499HK1EJKthqYQqxif/AFVpZRgSLNBELCwtrQQwqqNho5UulARvBSrCS4Au5SpMiQREbQbATLIyKRe6xmFk4wkVMvOVAnk81N7uzO1fc/dhZeIYSQfMe6Ng+DFwErtZxt4ADkiaB+8AZ2z/WuccMMF+v8YQaayPpIHCSkpA7AawCU3XMdmCxJue+BGZsPwbeAFO2J2z/rOd+qefdqfOOGLpkkUVs3EfbywCS3gJzti1pGdhPSfCdlTROiT1vMuJ6kqaBJeCe7e4f7tEBjtdxTyV9rf1HKUm8r2u+1DbKni1Q9iJ5VNsPgUFp1o3m2EJzn4hhS4GJ2Lhffe1e3+ce5Td1BXhue1Jlg7MXfeePAyv83TuRQUGBAmZtn9/k+EYz51XyPxAtyRJZxPCNAZ9re7rplDRGWVrrAHvq+5H1vKIufUk6Buyq/XPACUl767HdkvbVY1soqdBQlu3ma/sbsPMfvk/EpqTARAzfDeCapC6wta//JnDb9gfgNHC9KRQDXAI6khYpe3h8ArD9DrhA2QJ3CXhGidcH+A4ckrQAHAEu1/4HwN01L/kjWpe4/oj/hKQV2ztGPY+IRp5gIiKiFXmCiRghSaeAc2u6u7bPjmI+EcOUAhMREa3IEllERLQiBSYiIlqRAhMREa1IgYmIiFakwERERCt+A0MC8yztTO+kAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch2_1.best_score_, gsearch2_1.best_params_))\n",
    "test_means = gsearch2_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch2_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch2_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch2_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch2_1.cv_results_).to_csv('my_preds_maxdepth_min_child_weights_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(max_depth), len(min_child_weight))\n",
    "train_scores = np.array(train_means).reshape(len(max_depth), len(min_child_weight))\n",
    "\n",
    "for i, value in enumerate(max_depth):\n",
    "    pyplot.plot(min_child_weight, -test_scores[i], label= 'test_max_depth:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'max_depth' )                                                                                                      \n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig('max_depth_vs_min_child_weght_1.png' )"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
